{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a2d242c7-82fb-4ec2-b091-b25ab14133d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.optim.adamw import AdamW\n",
    "import numpy as np\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    AutoModelForSequenceClassification,\n",
    "    Trainer,\n",
    "    DataCollatorWithPadding,\n",
    "    TrainingArguments)\n",
    "from datasets import load_dataset\n",
    "import platform"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3905be32-c06d-4393-89ae-c5180a3d22d6",
   "metadata": {},
   "source": [
    "### HF_HOME这个环境变量的作用是缓存，如果想直接从磁盘上读取模型，还是得用完整路径，就像下面这样："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "18e87051-c432-4abc-b3df-e0cf260cbb21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如果报错ValueError: Invalid pattern: '**' can only be an entire path component，升级datasets:\n",
    "# pip install -U datasets\n",
    "# HF_HOME 可以自定义缓存文件夹\n",
    "if platform.system() == 'Windows':\n",
    "    model_path = 'E:/ai/huggingface-models/distilbert-base-uncased-finetuned-sst-2-english/'\n",
    "    data_path = \"E:/ai/data/glue/mrpc\"\n",
    "else:\n",
    "    model_path = '/home/will/huggingface-models/distilert-base-uncased-finetuned-sst-2-english/'\n",
    "    data_path = '/home/will/glue/mrpc'\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1d8c2260-7467-426f-a756-3e97877380a7",
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Incorrect path_or_model_id: 'E:/ai/huggingface-models/distilbert-base-uncased-finetuned-sst-2-english/'. Please provide either the path to a local folder or the repo_id of a model on the Hub.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mHFValidationError\u001b[0m                         Traceback (most recent call last)",
      "File \u001b[1;32md:\\tools\\Python38\\lib\\site-packages\\transformers\\utils\\hub.py:403\u001b[0m, in \u001b[0;36mcached_file\u001b[1;34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[0;32m    401\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m    402\u001b[0m     \u001b[39m# Load from URL or cache if already cached\u001b[39;00m\n\u001b[1;32m--> 403\u001b[0m     resolved_file \u001b[39m=\u001b[39m hf_hub_download(\n\u001b[0;32m    404\u001b[0m         path_or_repo_id,\n\u001b[0;32m    405\u001b[0m         filename,\n\u001b[0;32m    406\u001b[0m         subfolder\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m \u001b[39mif\u001b[39;49;00m \u001b[39mlen\u001b[39;49m(subfolder) \u001b[39m==\u001b[39;49m \u001b[39m0\u001b[39;49m \u001b[39melse\u001b[39;49;00m subfolder,\n\u001b[0;32m    407\u001b[0m         repo_type\u001b[39m=\u001b[39;49mrepo_type,\n\u001b[0;32m    408\u001b[0m         revision\u001b[39m=\u001b[39;49mrevision,\n\u001b[0;32m    409\u001b[0m         cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[0;32m    410\u001b[0m         user_agent\u001b[39m=\u001b[39;49muser_agent,\n\u001b[0;32m    411\u001b[0m         force_download\u001b[39m=\u001b[39;49mforce_download,\n\u001b[0;32m    412\u001b[0m         proxies\u001b[39m=\u001b[39;49mproxies,\n\u001b[0;32m    413\u001b[0m         resume_download\u001b[39m=\u001b[39;49mresume_download,\n\u001b[0;32m    414\u001b[0m         token\u001b[39m=\u001b[39;49mtoken,\n\u001b[0;32m    415\u001b[0m         local_files_only\u001b[39m=\u001b[39;49mlocal_files_only,\n\u001b[0;32m    416\u001b[0m     )\n\u001b[0;32m    417\u001b[0m \u001b[39mexcept\u001b[39;00m GatedRepoError \u001b[39mas\u001b[39;00m e:\n",
      "File \u001b[1;32md:\\tools\\Python38\\lib\\site-packages\\huggingface_hub\\utils\\_validators.py:106\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.<locals>._inner_fn\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    105\u001b[0m \u001b[39mif\u001b[39;00m arg_name \u001b[39min\u001b[39;00m [\u001b[39m\"\u001b[39m\u001b[39mrepo_id\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mfrom_id\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mto_id\u001b[39m\u001b[39m\"\u001b[39m]:\n\u001b[1;32m--> 106\u001b[0m     validate_repo_id(arg_value)\n\u001b[0;32m    108\u001b[0m \u001b[39melif\u001b[39;00m arg_name \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mtoken\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mand\u001b[39;00m arg_value \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[1;32md:\\tools\\Python38\\lib\\site-packages\\huggingface_hub\\utils\\_validators.py:154\u001b[0m, in \u001b[0;36mvalidate_repo_id\u001b[1;34m(repo_id)\u001b[0m\n\u001b[0;32m    153\u001b[0m \u001b[39mif\u001b[39;00m repo_id\u001b[39m.\u001b[39mcount(\u001b[39m\"\u001b[39m\u001b[39m/\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m--> 154\u001b[0m     \u001b[39mraise\u001b[39;00m HFValidationError(\n\u001b[0;32m    155\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mRepo id must be in the form \u001b[39m\u001b[39m'\u001b[39m\u001b[39mrepo_name\u001b[39m\u001b[39m'\u001b[39m\u001b[39m or \u001b[39m\u001b[39m'\u001b[39m\u001b[39mnamespace/repo_name\u001b[39m\u001b[39m'\u001b[39m\u001b[39m:\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    156\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mrepo_id\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m. Use `repo_type` argument if needed.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    157\u001b[0m     )\n\u001b[0;32m    159\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m REPO_ID_REGEX\u001b[39m.\u001b[39mmatch(repo_id):\n",
      "\u001b[1;31mHFValidationError\u001b[0m: Repo id must be in the form 'repo_name' or 'namespace/repo_name': 'E:/ai/huggingface-models/distilbert-base-uncased-finetuned-sst-2-english/'. Use `repo_type` argument if needed.",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m \u001b[43mAutoTokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheckpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcheckpoint\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      2\u001b[0m model \u001b[38;5;241m=\u001b[39m AutoModelForSequenceClassification\u001b[38;5;241m.\u001b[39mfrom_pretrained(model_path, num_labels \u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n\u001b[0;32m      3\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m AdamW(model\u001b[38;5;241m.\u001b[39mparameters())\n",
      "File \u001b[1;32md:\\tools\\Python38\\lib\\site-packages\\transformers\\models\\auto\\tokenization_auto.py:857\u001b[0m, in \u001b[0;36mAutoTokenizer.from_pretrained\u001b[1;34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[0m\n\u001b[0;32m    854\u001b[0m     \u001b[39mreturn\u001b[39;00m tokenizer_class\u001b[39m.\u001b[39mfrom_pretrained(pretrained_model_name_or_path, \u001b[39m*\u001b[39minputs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    856\u001b[0m \u001b[39m# Next, let's try to use the tokenizer_config file to get the tokenizer class.\u001b[39;00m\n\u001b[1;32m--> 857\u001b[0m tokenizer_config \u001b[39m=\u001b[39m get_tokenizer_config(pretrained_model_name_or_path, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[0;32m    858\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39m_commit_hash\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m tokenizer_config:\n\u001b[0;32m    859\u001b[0m     kwargs[\u001b[39m\"\u001b[39m\u001b[39m_commit_hash\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m tokenizer_config[\u001b[39m\"\u001b[39m\u001b[39m_commit_hash\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[1;32md:\\tools\\Python38\\lib\\site-packages\\transformers\\models\\auto\\tokenization_auto.py:689\u001b[0m, in \u001b[0;36mget_tokenizer_config\u001b[1;34m(pretrained_model_name_or_path, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, **kwargs)\u001b[0m\n\u001b[0;32m    686\u001b[0m     token \u001b[39m=\u001b[39m use_auth_token\n\u001b[0;32m    688\u001b[0m commit_hash \u001b[39m=\u001b[39m kwargs\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39m_commit_hash\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m--> 689\u001b[0m resolved_config_file \u001b[39m=\u001b[39m cached_file(\n\u001b[0;32m    690\u001b[0m     pretrained_model_name_or_path,\n\u001b[0;32m    691\u001b[0m     TOKENIZER_CONFIG_FILE,\n\u001b[0;32m    692\u001b[0m     cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[0;32m    693\u001b[0m     force_download\u001b[39m=\u001b[39;49mforce_download,\n\u001b[0;32m    694\u001b[0m     resume_download\u001b[39m=\u001b[39;49mresume_download,\n\u001b[0;32m    695\u001b[0m     proxies\u001b[39m=\u001b[39;49mproxies,\n\u001b[0;32m    696\u001b[0m     token\u001b[39m=\u001b[39;49mtoken,\n\u001b[0;32m    697\u001b[0m     revision\u001b[39m=\u001b[39;49mrevision,\n\u001b[0;32m    698\u001b[0m     local_files_only\u001b[39m=\u001b[39;49mlocal_files_only,\n\u001b[0;32m    699\u001b[0m     subfolder\u001b[39m=\u001b[39;49msubfolder,\n\u001b[0;32m    700\u001b[0m     _raise_exceptions_for_gated_repo\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m    701\u001b[0m     _raise_exceptions_for_missing_entries\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m    702\u001b[0m     _raise_exceptions_for_connection_errors\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m    703\u001b[0m     _commit_hash\u001b[39m=\u001b[39;49mcommit_hash,\n\u001b[0;32m    704\u001b[0m )\n\u001b[0;32m    705\u001b[0m \u001b[39mif\u001b[39;00m resolved_config_file \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m    706\u001b[0m     logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mCould not locate the tokenizer configuration file, will try to use the model config instead.\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[1;32md:\\tools\\Python38\\lib\\site-packages\\transformers\\utils\\hub.py:469\u001b[0m, in \u001b[0;36mcached_file\u001b[1;34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs)\u001b[0m\n\u001b[0;32m    467\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mEnvironmentError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mThere was a specific connection error when trying to load \u001b[39m\u001b[39m{\u001b[39;00mpath_or_repo_id\u001b[39m}\u001b[39;00m\u001b[39m:\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m{\u001b[39;00merr\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[0;32m    468\u001b[0m \u001b[39mexcept\u001b[39;00m HFValidationError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m--> 469\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mEnvironmentError\u001b[39;00m(\n\u001b[0;32m    470\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mIncorrect path_or_model_id: \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mpath_or_repo_id\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m. Please provide either the path to a local folder or the repo_id of a model on the Hub.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    471\u001b[0m     ) \u001b[39mfrom\u001b[39;00m\u001b[39m \u001b[39m\u001b[39me\u001b[39;00m\n\u001b[0;32m    472\u001b[0m \u001b[39mreturn\u001b[39;00m resolved_file\n",
      "\u001b[1;31mOSError\u001b[0m: Incorrect path_or_model_id: 'E:/ai/huggingface-models/distilbert-base-uncased-finetuned-sst-2-english/'. Please provide either the path to a local folder or the repo_id of a model on the Hub."
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_path, checkpoint=checkpoint)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_path, num_labels =2)\n",
    "optimizer = AdamW(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ee70782a-ffbb-427f-8469-c4a4b820d2d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = tokenizer(\"i've been waiting for a huggingface course for too long\", \"This course is amazing!\")\n",
    "\n",
    "# <class 'transformers.tokenization_utils_base.BatchEncoding'>\n",
    "print(f'type(batch) = {type(batch)}')\n",
    "\n",
    "batch['labels'] = torch.tensor([1, 1])\n",
    "print(f'batch = {batch}')\n",
    "print(f\"batch.input_ids = {batch['input_ids']}\")\n",
    "\n",
    "tokenized_sentences = tokenizer.convert_ids_to_tokens(batch['input_ids'])\n",
    "print(f'将生成的ids还原: {tokenized_sentences}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d55baa22-95ff-4184-acc3-1f29114a2451",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_datasets = load_dataset(path=data_path)\n",
    "print(raw_datasets)\n",
    "\n",
    "# tokenized_dataset = tokenizer(raw_datasets[\"train\"][\"sentence1\"],\n",
    "#                               raw_datasets[\"train\"][\"sentence2\"],\n",
    "#                               padding=True,\n",
    "#                               truncation=True)\n",
    "# print(tokenized_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ef6ea709-5fb4-4193-934a-5665cb77adfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_function(example):\n",
    "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
    "\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)\n",
    "print(tokenized_datasets)\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
    "\n",
    "samples = tokenized_datasets[\"train\"][:32]\n",
    "samples = {k: v for k, v in samples.items() if k not in [\"idx\", \"sentence1\", \"sentence2\"]}\n",
    "lens = [len(x) for x in samples[\"input_ids\"]]\n",
    "print(lens)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9b9b1deb-cba2-4889-91cb-f57503346a8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "batch = data_collator(samples)\n",
    "print(f'batch after DataCollatorWithPadding: {batch}')\n",
    "dict1 = {k: v.shape for k, v in batch.items()}\n",
    "print(dict1)\n",
    "\n",
    "training_args = TrainingArguments('test-trainer')\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    training_args,\n",
    "    train_dataset=tokenized_datasets['train'],\n",
    "    eval_dataset=tokenized_datasets['validation'],\n",
    "    data_collator=data_collator, # 可以省略\n",
    "    processing_class=tokenizer,\n",
    ")\n",
    "trainer.train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3e7e79f2-9eba-4558-8328-1be606141f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = trainer.predict(tokenized_datasets['validation'])\n",
    "print(predictions.predictions.shape, predictions.label_ids.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e0b2079f-e623-4245-b0c8-6b1b6e2ec1f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = np.argmax(predictions.predictions, axis=-1)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dde12d6b-1692-46d9-bcf8-17893309395e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "\n",
    "metric = evaluate.load('glue', 'mrpc')\n",
    "metric.compute(predictions=preds, references = predictions.label_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "aed3abb1-123b-47c2-8ae7-4557f33d0f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(eval_preds):\n",
    "    metric = evaluate.load('glue', 'mrpc')\n",
    "    logits, labels = eval_preds\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6d492919-cad5-4795-8d01-86e1b3563d48",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments('test-trainer', eval_strategy='epoch')\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_path, num_labels =2)\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    training_args,\n",
    "    train_dataset=tokenized_datasets['train'],\n",
    "    eval_dataset=tokenized_datasets['validation'],\n",
    "    data_collator=data_collator, # 可以省略\n",
    "    processing_class=tokenizer,\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "trainer.train()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
